homeq 1.1.4

data/lib/homeq/sobs/server.rb

@@ -0,0 +1,654 @@
+#############################################################################
+#
+# $Id: server.rb 48 2008-11-19 05:11:59Z colin $
+#
+# Author:: Colin Steele (colin@colinsteele.org)
+# Homepage::  
+# 
+# TODO: info 
+#
+#----------------------------------------------------------------------------
+#
+#      Copyright (C) 2008 by Colin Steele. All Rights Reserved.
+#                        colin@colinsteele.org
+# 
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of either: 1) the GNU General Public License
+# as published by the Free Software Foundation; either version 2 of the
+# License, or (at your option) any later version; or 2) Ruby's License.
+# 
+# See the file COPYING for complete licensing information.
+#
+#---------------------------------------------------------------------------
+#
+# 
+#############################################################################
+
+module HomeQ
+  
+  module SOBS
+
+    ######################################################################
+    # Server Job
+    ######################################################################
+    # 
+    # This class is the server-side representation of a message, or a
+    # "job" - a piece of work that somebody sent, that somebody will
+    # eventually pick up and do something with.
+    # 
+    # On the server side, the job lifecycle looks like this:
+    # 
+    #    put with delay               release with delay
+    #   ----------------> [DELAYED] <------------.
+    #                         |                  |
+    #                         |(time passes)     |
+    #                         |                  |
+    #   put                   v     reserve      |        delete
+    #   -----------------> [READY] ---------> [RESERVED] --------> *poof*
+    #                        ^ ^                 | |
+    #                        |  \ release        | |            
+    #                        |   `---------------' |        
+    #                        |                     |                        
+    #                        | kick                |                   
+    #                        |                     |                        
+    #                        |                bury |                   
+    #                    [BURIED] <----------------' 
+    #                        |                          
+    #                        | delete                   
+    #                        `--------> *poof*          
+    #
+    # In the simplest case, jobs go from start to ready to reserved to
+    # *poof*.
+    #
+    class ServerJob < SOBS::Job
+
+      # The Foreman watches jobs
+      include Observable
+
+      # We have to carp.
+      include Base::Logging
+
+      # Object pool
+      include Poolable
+      pool_init(10000)
+
+      # Time To Run
+      attr :ttr, true
+      
+      # How important?  Not implemented.
+      attr :priority, true
+      
+      # Delay N seconds before ready
+      attr :delay, true
+      
+      # Connection working on this job
+      attr :reserving_connection 
+
+      # A count of the total number of jobs that have had a life in
+      # the system.
+      @@job_count = 0
+
+      # Create a new ServerJob.  Called by Foreman#create_job.
+      # A Job is based on receipt of a message, from a source, and it
+      # is managed by a foreman.  The message is assumed to have three
+      # args: priority (ignored), delay, and ttr (time to run).
+      def initialize(message, source)
+        logger.debug4 {
+          'INIT'
+        }
+        super(message, source)
+        @@job_count     += 1
+        @priority, @delay, @ttr = @message.args.collect { |arg|
+          arg.to_i
+        }
+
+        # Create a state machine to drive.
+        @state = Statemachine.build do
+          state :start do
+            event :put,                 :ready
+            event :put_with_delay,      :delayed
+          end
+          state :delayed do
+            on_entry :start_delay_timer
+            event :delay_timeout,       :ready
+          end
+          state :ready do
+            on_entry :broadcast_change
+            event :reserve,             :reserved
+          end
+          state :reserved do
+            on_entry :start_run_timer
+            event :delete,              :deleted
+            event :release,             :ready
+            event :run_timeout,         :ready
+            event :release_with_delay,  :delayed
+            event :bury,                :buried
+            on_exit :exit_reserved_state
+          end
+          state :buried do
+            on_entry :broadcast_change
+            event :delete,              :deleted
+            event :kick,                :ready
+          end
+          state :deleted do
+            on_entry :broadcast_change
+          end
+        end
+        @state.context = self
+      end
+
+      def deinitialize
+        logger.debug4 {
+          'DEINIT'
+        }
+        # first take care of superclass's vars
+        @message = @queue = @job_id = nil
+        
+        # now ours
+        instance_variables.each {|v|
+          next if v == "@state"     # keeper.  largely the point of recycling
+          next if v =~ /observer/   # keeper.  our observer(s)
+          eval "#{v} = nil"
+        }
+        @state.state = :start
+      end
+
+      def reinitialize(message, source)
+        # Call super's initialize().
+        SOBS::Job.instance_method(:initialize).bind(self).call(message,source)
+        @@job_count     += 1
+        @priority, @delay, @ttr = @message.args.collect { |arg|
+          arg.to_i
+        }
+        logger.debug4 {
+          "REINIT #{@state.state}"
+        }
+      end
+      
+      def run
+        logger.debug4 {
+          'LAUNCH'
+        }
+        broadcast_change
+        if delay > 0
+          put_with_delay
+        else
+          put
+        end
+      end
+      
+      # Takes this job from the start state to ready.
+      def put
+        @state.put
+      end
+
+      # Takes this job from the start state to delayed.  As a
+      # consequence, with kick off a timer that will move this job
+      # from delayed to ready when it expires.
+      def put_with_delay
+        @state.put_with_delay
+      end
+
+      # Delete this job (*poof*).  A 'deleted' reply is sent to the
+      # connection that sent the delete (which may or may not be the
+      # connection that reserved the job in the first place).
+      def delete(conn)
+        conn.deleted(job_id) # may or may not be @reserving_connection
+        @state.delete
+      end
+
+      # Mark this job as being processed.  The reserving connection is
+      # sent a 'reserved' reply along with the contents of this job.
+      # We store a notion of the reserving connection, and push this
+      # job from the 'ready' state to the 'reserved' state.
+      def reserve(conn)
+        @state.reserve
+        conn.reserved(job_id, payload)
+        @reserving_connection = conn
+      end
+
+      # Take this job out of the "being worked on" state and back to
+      # availability in the 'ready' state.
+      def release
+        @state.release
+      end
+
+      # Take this job out of the "being worked on" state and back to
+      # availability in the 'ready' state, but after going through a
+      # 'delayed' state.  We kick off a timer that moves the job from
+      # 'delayed' to 'ready' when it expires.
+      def release_with_delay(priority, delay)
+        @priority = priority
+        @delay = delay
+        @state.release_with_delay
+      end
+
+      # Push this job off to the side so it is not available to be
+      # worked on.
+      def bury(conn)
+        conn.buried(job_id) # may or may not be @reserving_connection
+        @state.bury
+      end
+
+      # Move this job from the 'buried' state (see #bury, above) to
+      # the 'ready' state.
+      def kick
+        @state.kick
+      end
+
+      protected
+
+      def start_delay_timer
+        broadcast_change
+        @delay_timer = EventMachine::Timer.new(@delay.to_i) do
+          @delay_timer = nil
+          @state.delay_timeout
+        end
+      end
+      
+      def start_run_timer
+        broadcast_change
+        return if @ttr < 1
+        start_deadline_soon_timer
+        @run_timer = EventMachine::Timer.new(@ttr.to_i) do
+          logger.info {
+            peer = @reserving_connection.remote_endpoint
+            "Job #{job_id} run timeout on SOBS Peer #{peer}."
+          }
+          @run_timer = nil
+          @state.run_timeout
+        end
+      end
+
+      def start_deadline_soon_timer
+        @deadline_soon_timer = EventMachine::Timer.new(@ttr - 1) do
+          @deadline_soon_timer = nil
+          if !@reserving_connection
+            msg = "Logic error: nil reserving connection."
+            Base::System.instance.die(msg)
+          end
+          if @reserving_connection.state.state == :open
+            @reserving_connection.deadline_soon(job_id)
+          end
+        end
+      end
+      
+      def exit_reserved_state
+        @run_timer.cancel if @run_timer
+        @deadline_soon_timer.cancel if @deadline_soon_timer
+        @deadline_soon_timer    = nil
+        @run_timer              = nil
+        @reserving_connection   = nil
+      end
+
+      def broadcast_change
+        changed
+        notify_observers(self, @state.state)
+      end
+      
+    end
+
+
+    ######################################################################
+    # Server Connection
+    ######################################################################
+    # 
+    # This class represents the server-side of a connection between a
+    # SOBS client and the SOBS server.
+    # 
+    class ServerConnection < SOBS::Connection
+
+      # This connection's state
+      attr :server_state, false
+
+      # Job handler
+      attr :handler, true
+
+      # Create a new ServerConnection, which is a child of SOBS::Connection.
+      # Called from Server#start via EventMachine. When an incoming
+      # TCP connection is received, a ServerConnection is created to
+      # handle it.
+      def initialize
+        super
+
+        # The state machine that drives our behavior.
+        
+        @server_state = Statemachine.build do
+          state :start do
+            event :open, :open
+            event :close, :closed
+          end
+          state :waiting_for_job do
+            on_entry :reserve_ready_job
+            event :sent_job, :open
+            event :reserve, :waiting_for_job
+            event :received_job, :waiting_for_job, :reserve_ready_job
+            event :subscribe, :waiting_for_job
+            event :unsubscribe, :open
+            event :close, :closed
+          end
+          state :open do
+            event :subscribe, :waiting_for_job
+            event :unsubscribe, :open
+            event :reserve, :waiting_for_job
+            event :received_job, :open
+            event :close, :closed
+          end
+          state :closed do
+            event :close, :closed
+          end
+        end
+        @server_state.context = self
+
+        # stats
+        @jobs_received                          = 0
+
+        # An attribute of this connection.  If true, the client has
+        # indicated that they want to be sent a ready job whenever
+        # it's available.  Otherwise, they must ask for each job via
+        # the 'reserve' command.
+        @subscribed                             = nil
+      end
+
+      #--
+      # SOBS::Connection entry points
+      #++
+
+      # Called from the lower-level Connection abstraction when a
+      # client connection is complete and ready to use.
+      def opened
+        @server_state.open
+      end
+      
+      # Called from the lower-level Connection abstraction when a
+      # client connection is dead.
+      def closed
+        logger.info {
+          "Job pool size: #{ServerJob.pool.size}"
+        }
+        @server_state.close
+      end
+
+      
+      #--
+      # High level message interface - sending
+      #++
+
+      # Send a 'reserved' message to the remote endpoint, indicating
+      # that it's now in charge of doing the indicated piece of work.
+      # Track the server state appropriately (switch state to 'open'
+      # unless the remote endpoint has subscribed).
+      def reserved(job_id, payload)
+        super(job_id, payload) # send the msg to the client
+        @server_state.sent_job unless @subscribed
+      end
+
+      #--
+      # High level message interface - receiving
+      #++
+
+      # Handle a 'put' message from the remote endoint.  We create a
+      # message and reply to let the other side know we got it.
+      def receive_put(message)
+        if !ok_to_receive?
+          logger.error {
+            "Connection not ready for 'put'"
+          }
+          close
+          return
+        end
+        if message.args.length != 5
+          logger.error {
+            "Malformed put from SOBS peer #{remote_endpoint}.  " +
+              "Closing connection."
+          }
+          close
+          return
+        end
+        create_job(message)
+        @server_state.received_job
+      end
+
+      # Handle a 'reserve' message from the client, which says that
+      # they're ready for work to be sent to them.
+      def receive_reserve(message)
+        if !ok_to_receive?
+          logger.error {
+            "Connction not ready to 'receive'"
+          }
+          return
+        end
+        @server_state.reserve
+      end
+
+      # Handle a 'delete' message from the client, which says that
+      # they've completed the work.  We let the foreman know.
+      def receive_delete(message)
+        ok_to_receive? || return
+        @server.foreman.delete_job(self, message.args[0])
+      end
+
+      # Handle a 'release' message from the client, which says that
+      # they're not going to finish (or delete) the work, and to make
+      # it available for someone else to work on.
+      def receive_release(message)
+        ok_to_receive? || return
+        @server.foreman.release_job(self,
+                                    message.args[0],
+                                    message.args[1],
+                                    message.args[2])
+      end
+      
+      # Handle a 'bury' message from the client, which says that
+      # the job shouldn't be available for anyone.
+      def receive_bury(message)
+        ok_to_receive? || return
+        @server.foreman.bury_job(self, message.args[0])
+      end
+
+      # Handle a 'kick' message from the client, which move some
+      # number of buried jobs to the 'ready' state.
+      def receive_kick(message)
+        ok_to_receive? || return
+        @server.foreman.kick(self, message.args[0])
+      end
+
+      # Handle a 'subscribe' message from the client, which tells us
+      # that they want to be issued work to do whenever it's available.
+      def receive_subscribe(message)
+        @subscribed = true
+        @server_state.subscribe
+      end
+
+      # Handle a 'unsubscribe' message from the client, which tells us
+      # that they want to send explicit requests for more work via
+      # 'reserve' commands.
+      def receive_unsubscribed(message)
+        @subscribed = false
+        @server_state.unsubscribed
+      end
+
+      #--
+      # Other
+      #++
+
+      def open
+        super
+        @server_state.open
+      end
+
+      def unknown_message(command_name)
+        logger.info {
+          "SOBS peer #{remote_endpoint} sent unknown/unsupported " +
+            "command: #{command_name}"
+        }
+        close
+      end
+
+      def release_job(job)
+      end
+
+      def to_s
+        str = ""
+        str << "#{self.class} state: #{server_state.state} "
+        str << "subscribed?: #{@subscribed ? 'Y' : 'N'}\n"
+        str << "jobs in: #{@jobs_received} "
+        str << "job pool size: #{ServerJob.pool ? ServerJob.pool.size : '?'}\n"
+        str << super.gsub(/\n/m, "\n  ")
+      end
+
+      protected
+
+      def reserve_ready_job
+        @server.foreman.dole_out_jobs
+      end
+
+      def create_job(message)
+        logger.debug4 {
+          "Server handler: #{@handler}"
+        }
+        if @handler
+          if @handler.respond_to?(:call)
+            @handler.call(message, self)
+          elsif @handler.respond_to?(:create_job)
+            @handler.create_job(message, self)
+          elsif @handler.respond_to?(:new)
+            job = @handler.new(message, self)
+            if job.respond_to?(:run)
+              job.run
+            end
+          end
+        else
+          @jobs_received += 1
+          @server.foreman.create_job(message, self)
+        end
+      end
+
+      def get_next_message
+        reserve
+      end
+
+    end # ServerConnection
+
+    
+    ######################################################################
+    # Server / Listener class
+    ######################################################################
+
+    class Server
+
+      DEFAULT_PORT = 56000
+      DEFAULT_HOST = 'localhost'
+
+      include Base::Logging
+      include Base::Configuration
+
+      attr_accessor :connections
+      attr :foreman, true
+      attr :queue_name, true
+      attr :total_connections, false
+
+      # Make our config available
+      module HomeQ::Base::Commando::InstanceMethods
+        config_accessor :queue_name
+      end
+      
+      def self.create_home_queue(queuename, handler=nil)
+        logger = HomeQ::Base::Logging::Logger.instance.logger
+        config = HomeQ::Base::Configuration::Configuration.instance
+        if queuename =~ /^__/
+          return
+        elsif !config.topology[queuename]
+          Base::System.instance.die("No queue info for '#{queuename}'")
+        end
+        logger.info {
+          "Starting home queue '#{queuename}'" +
+          (handler ? ", handler: #{handler}" : '')
+        }
+        q, host, port, hostname = config.topology[queuename]
+        server = HomeQ::SOBS::Server.new(host, port, queuename, handler)
+        Base::System.instance.servers << server
+      end
+
+      def initialize(host=nil, port=nil, queuename=nil, handler=nil)
+        @host 			= host
+        @port 			= port
+        @connections 		= []
+        @foreman 		= nil
+        @queue_name 		= queuename
+        @total_connections 	= 0
+        @started_at		= nil
+        @stopped_at		= nil
+        @handler		= handler
+      end
+
+      def start
+        @started_at		= Time.now
+        @foreman 		||= Foreman.new(self)
+        begin
+          @signature =
+            EventMachine.start_server(@host,
+                                      @port,
+                                      HomeQ::SOBS::ServerConnection) { |conn|
+            conn.server  = self
+            conn.handler = @handler
+            @connections << conn
+            @total_connections += 1
+          }
+        rescue Exception => e
+          msg = "Error starting SOBS listener '#{@host}:#{@port}'." +  
+            "Probably a config error. Original error: #{e.message}."
+          Base::System.instance.die(msg)
+        else
+          logger.info {
+            "SOBS server running on #{@host}:#{@port}"
+          }
+        end
+      end
+
+      def stop
+        @stopped_at		= Time.now
+        EventMachine.stop_server(@signature)
+        unless wait_for_connections
+          # Still some running
+          EventMachine.add_periodic_timer(1) {
+            wait_for_connections
+          }
+        end
+      end
+
+      def to_s
+        str = ''
+        str << "#{self.class} Queuename: #{@queue_name}"
+        str << " on #{@host}:#{@port}" if @host || @port
+        str << "\n"
+        str << "Started: #{@started_at} (#{Time.now - @started_at}s)\n"
+        str << "Total connections: #{@total_connections} "
+        str << "Active connections: #{connections.length}"
+        str << ("\n" + connections.join("\n")).gsub(/\n/m, "\n  ")
+        str << ("\n" + @foreman.to_s).gsub(/\n/m, "\n  ") if @foreman
+        str
+      end
+
+      protected
+
+      def wait_for_connections
+        if @connections.empty?
+          true
+        else
+          logger.info {
+            "Waiting for #{@connections.length} connections to finish."
+          }
+          @connections.each { |c|
+            c.close
+          }
+          false
+        end
+      end
+      
+    end # Server
+
+  end # SOBS
+
+end # HomeQ
+

data/lib/homeq/sobs/sobs.rb

@@ -0,0 +1,45 @@
+#############################################################################
+#
+# $Id: sobs.rb 42 2008-10-24 05:53:35Z colin $
+#
+# Author:: Colin Steele (colin@colinsteele.org)
+# Homepage::  
+# 
+# TODO: info 
+#
+#----------------------------------------------------------------------------
+#
+#      Copyright (C) 2008 by Colin Steele. All Rights Reserved.
+#                        colin@colinsteele.org
+# 
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of either: 1) the GNU General Public License
+# as published by the Free Software Foundation; either version 2 of the
+# License, or (at your option) any later version; or 2) Ruby's License.
+# 
+# See the file COPYING for complete licensing information.
+#
+#---------------------------------------------------------------------------
+#
+# 
+#############################################################################
+
+require 'statemachine'
+require 'observer'
+require 'uuidtools'
+
+require 'homeq/sobs/sender'
+require 'homeq/sobs/connection'
+require 'homeq/sobs/client'
+require 'homeq/sobs/message'
+require 'homeq/sobs/job'
+require 'homeq/sobs/server'
+require 'homeq/sobs/topology'
+require 'homeq/sobs/queue'
+require 'homeq/sobs/foreman'
+
+module HomeQ
+  module SOBS
+    PROTOCOL_VERSION = 1
+  end
+end