ap4r 0.3.1 → 0.3.2

data/lib/ap4r/message_store_ext.rb

@@ -1,9 +1,9 @@
 # Author:: Shunichi Shinohara
-# Copyright:: Copyright (c) 2006 Future System Consulting Corp.
+# Copyright:: Copyright (c) 2007 Future Architect Inc.
 # Licence:: MIT Licence
 
-module ReliableMsg
-  module MessageStore
+module ReliableMsg #:nodoc:
+  module MessageStore #:nodoc:
     class Base
 
       cattr_accessor :use_mysql_extention
@@ -37,11 +37,15 @@ module ReliableMsg
 end
 
 if ReliableMsg::MessageStore::Base.use_mysql_extention
-  class Mysql
+  class Mysql #:nodoc:
     alias original_query query
+    
+    # In Ruby/MySQL, +query+ method does NOT care about a given block.
+    # To make it behave the same as MySQL/Ruby, this method adds iteration
+    # over query results.
     def query(q, &block)
       maybe_result = original_query(q, &block)
-      puts "Mysql extention: query called by #{q}"
+      puts "Mysql extention: query called by #{q}" if $DEBUG
       puts "Mysql#query returns #{maybe_result}(class: #{maybe_result.class})." if $DEBUG
       return maybe_result unless block && maybe_result.kind_of?(Mysql::Result)
       begin

data/lib/ap4r/mongrel.rb

@@ -1,5 +1,5 @@
 # Author:: Shunichi Shinohara
-# Copyright:: Copyright (c) 2007 Future Architect Corp.
+# Copyright:: Copyright (c) 2007 Future Architect Inc.
 # Licence:: MIT Licence
 
 require 'mongrel'
@@ -8,8 +8,10 @@ require 'cgi'
 require 'ap4r'
 
 module Ap4r
-  module Mongrel
 
+  module Mongrel #:nodoc:
+
+    # Gather controls AP4R server.
     class Ap4rConfigurator < ::Mongrel::Configurator
 
       def stop(needs_restart=false)
@@ -39,6 +41,7 @@ module Ap4r
 
     # Implements a handler that can run AP4R.
     # * If the requested exact PATH_INFO exists as a file then serve it.
+    # + (Second, access server information or queue/topic API) NOT IMPLEMENTED.
     # * Finally, raise an exception.
     #
     # memo: want to use this handler to take information from AP4R server
@@ -47,11 +50,13 @@ module Ap4r
     # TODO not yet implemented 2007/04/09 by shino
     #
     class Ap4rHandler < ::Mongrel::HttpHandler
-      STDERR.puts "CAUTION! This script is rather experimental."
       attr_reader :files
       @@file_only_methods = ["GET","HEAD"]
 
       def initialize(options)
+        # TODO: needs various modes for easy operations 2007/05/02 by shino
+        #        e.g. not to initialize message store, not to start dispatchers, etc. 
+
         # TODO what is "false" here? 2007/04/13 by shinohara
         @files = ::Mongrel::DirHandler.new(options[:docroot], false)
         @tick = Time.now
@@ -89,7 +94,7 @@ module Ap4r
       def reload!
         begin
           #TODO not implemented 2007/04/09 by shino
-          raise "not yet implemented!."
+          raise "not yet implemented!"
         end
       end
 

data/{script → lib/ap4r}/mongrel_ap4r.rb

@@ -86,8 +86,7 @@ module Ap4r::Mongrel
 
       valid_dir? File.dirname(@log_file), "Path to log file not valid: #@log_file"
       valid_dir? File.dirname(@pid_file), "Path to pid file not valid: #@pid_file"
-      # TODO: now not utilized, skip checking 2007/04/23 by shino
-      #valid_dir? @docroot, "Path to docroot not valid: #@docroot"
+      valid_dir? @docroot, "Path to docroot not valid: #@docroot"
 
       return @valid
     end

data/lib/ap4r/multi_queue.rb

@@ -1,8 +1,9 @@
 # Author:: Shunichi Shinohara
-# Copyright:: Copyright (c) 2006 Future System Consulting Corp.
+# Copyright:: Copyright (c) 2007 Future Architect Inc.
 # Licence:: MIT Licence
 
 module ReliableMsg
+
   # The +MultiQueue+ is a kind of clients.
   # This offers two extentions to <tt>ReliableMsg::Queue</tt>
   # 1. specify multiple target queues, by as comma-separated queue names.

data/lib/ap4r/queue_manager_ext.rb

@@ -1,31 +1,84 @@
 # Author:: Shunichi Shinohara
-# Copyright:: Copyright (c) 2006 Future System Consulting Corp.
+# Copyright:: Copyright (c) 2007 Future Architect Inc.
 # Licence:: MIT Licence
 
-require 'ap4r/message_store_ext'
-require 'ap4r/multi_queue'
-require 'ap4r/retention_history'
-
-require 'soap/wsdlDriver'
-require 'xmlrpc/client'
-require 'uri'
-require 'net/http'
-
 require 'active_support'
 require 'yaml'
 require 'thread'
 require 'pp'
 
 require 'uuid'
+require 'reliable-msg'
 
-module ReliableMsg
-  module LifecycleListener #:nodoc:
+require 'ap4r/message_store_ext'
+require 'ap4r/multi_queue'
+require 'ap4r/retention_history'
+require 'ap4r/dispatcher'
+require 'ap4r/carrier'
+
+module ReliableMsg #:nodoc:
+
+  # = Dynamic configuration with ERb
+  # 
+  # Some times you would like to inject dynamic values into your configuration file. 
+  # In these cases, you can mix ERb in with your YAML to code some logic, like:
+  # 
+  # <% acl = [] %>
+  # <% for i in 1..100 %>
+  # <% acl << "192.168.0.#{i}" %>
+  # <% end %> 
+  # acl: <%= acl.map{|ip| "allow #{ip}"}.join(' ')
+  #
+  class Config
     
+    alias :load_no_create_original :load_no_create
+    alias :load_or_create_original :load_or_create
+   
+    #--
+    # TODO: should enhance YAML.load_documents instead of this method?, 2007/5/7 kato-k
+    def load_no_create
+      if File.exist?(@file)
+        @config= {}
+        File.open @file, "r" do |input|
+          YAML.load_documents(erb_render(input.read)) do |doc|
+            @config.merge! doc
+          end
+        end
+        true
+      end
+    end
+    
+    #--
+    # TODO: should enhance YAML.load_documents instead of this method?, 2007/5/7 kato-k
+    def load_or_create
+      if File.exist?(@file)
+        @config= {}
+        File.open @file, "r" do |input|
+          YAML.load_documents(erb_render(input.read)) do |doc|
+            @config.merge! doc
+          end
+        end
+        @logger.info format(INFO_LOADED_CONFIG, @file)
+      else
+        @config = {
+          "store" => DEFAULT_STORE,
+          "drb" => DEFAULT_DRB
+        }
+        save
+        @logger.info format(INFO_CREATED_CONFIG, @file)
+      end
+    end
+  
+    private
+    def erb_render(configuration_content)
+      ERB.new(configuration_content).result
+    end
   end
-
+  
   class QueueManager
-    # Gets a queue name which has the most stale message
-    # in queues specified by +multi_queue+.
+
+    # Gets a queue name which has the most stale message.
+    # +multi_queue+ specifies the target queue names to search.
     def stale_queue multi_queue
       @store.stale_queue multi_queue
     end
@@ -37,7 +90,6 @@ module ReliableMsg
     # Hooks original initialize method to add lifecyle listeners.
     #--
     # TODO: Make dispatchers and carriers lifecyle listeners, 2006/09/01 shino
-    # TODO: and separate them from QueueManager, 2006/09/01 shino
     def initialize options = nil #:notnew:
       initialize_original options
       @global_lock ||= Mutex.new
@@ -51,14 +103,25 @@ module ReliableMsg
       self.class.class_eval("attr_#{attr_mode} :#{iv_name}")
     end
 
+    # Starts reliable-msg server and something around it.
+    #
+    # Order is:
+    # 1. Original reliable-msg server (message store and druby).
+    # 2. Dispatchers
+    # 3. Carriors (if exists)
+    # These are Reversed in +stop+.
     def start
       begin
         @global_lock.synchronize do
           return if @@active == self
-          @dispatch_targets = ''
           start_original
-          start_dispatchers
-          start_carriers
+
+          @dispatchers = ::Ap4r::Dispatchers.new(self, @config.dispatchers, @logger)
+          @dispatchers.start
+
+          @carriors = ::Ap4r::Carriers.new(self, @config.carriers, @logger, @dispatchers)
+          @carriors.start
+
           @lifecycle_listeners.each {|l| l.start }
         end
       rescue Exception => err
@@ -67,195 +130,18 @@ module ReliableMsg
       end
     end
 
+    # Stops reliable-msg server and something around it.
+    # See +start+ also.
     def stop
       @global_lock.synchronize do
         return unless @@active == self
         @lifecycle_listeners.each {|l| l.stop }
-        stop_carriers
-        stop_dispatchers
+        @carriors.stop
+        @dispatchers.stop
         stop_original
       end
     end
 
-    def start_dispatchers
-      begin
-        return unless @config.dispatchers
-        @logger.info{ "about to start dispatchers with config #{@config.dispatchers.to_yaml}" }
-        @disps = ThreadGroup.new
-        @config.dispatchers.each{ |conf|
-          conf["threads"].to_i.times { |index|
-            Thread.fork(@disps, conf["targets"], index){|group, targets, index|
-              dispatch_loop(group, targets, index)
-            }
-          }
-          @dispatch_targets.concat(conf["targets"]).concat(';')
-          @logger.info{ "dispatch targets are : #{@dispatch_targets}" }
-        }
-        @logger.info "queue manager has forked dispatchers"
-      rescue Exception => err
-        @logger.warn{"Error in starting dipatchers #{err}"}
-        @logger.warn{err.backtrace.join("\n")}
-        raise err
-      end
-    end
-
-    def dispatch_loop(group, targets, index)
-      group.add Thread.current
-      mq = MultiQueue.new targets
-      @logger.info{ "start dispatcher: targets= #{mq} (index #{index})" }
-      until Thread.current[:dying]
-        sleep 0.1
-        # @logger.debug{ "try dispatch #{mq} #{mq.name}" }
-        # TODO: needs timeout?, 2006/10/16 shino
-        begin
-          mq.get{|m|
-            @logger.debug{"dispatcher get message\n#{m.to_yaml}"} if m
-            response = 
-            case m.headers[:dispatch_mode]
-            when :HTTP
-              dispatch_via_http(m)
-            when :XMLRPC
-              dispatch_via_xmlrpc(m)
-            when :SOAP
-              dispatch_via_soap(m)
-            else
-              raise "undefined dispatch mode #{m.headers[:mode]}"
-            end
-            @logger.debug{"dispatcher get response\n#{response.to_yaml}"}
-          }
-        rescue Exception => err
-          @logger.warn("dispatch err #{err.inspect}")
-          @logger.warn(err.backtrace.join("\n"))
-        end
-      end
-      @logger.info{"end dispatcher #{mq} (index #{index})"}
-    end
-
-    # Dispatch via a HTTP protocol
-    # Current implementation uses POST method, irrespectively options[:target_method]
-    #
-    # Determination of "success" is two fold
-    # * status code should be 200, other codes (including 201-2xx) are treated as error
-    # * when status code is 200, body should include a string "true"
-    def dispatch_via_http(message)
-      #TODO: Add some request headers e.g. User-Agent and Accept, 2006/10/12 shino
-      #TODO: Now support POST only, 2006/10/12 shino
-      response = Net::HTTP.post_form(URI.parse(message[:target_url]),
-                                     message.object)
-      @logger.debug{"response status [#{response.code} #{response.message}]"}
-
-      #TODO: make the difinition of success variable, 2006/10/13 shino
-      unless response.kind_of?(Net::HTTPOK)
-        error_message = "HTTP Response FAILURE, status [#{response.code} #{response.message}]"
-        @logger.error(error_message)
-        @logger.info{response.to_yaml}
-        #TODO: must create AP4R specific Exception class, 2006/10/12 shino
-        raise StandardError.new(error_message)
-      end
- 
-      unless response.body =~ /true/
-        error_message = "HTTP Response FAILURE, status [#{response.code} #{response.message}], body [#{response.body}]"
-        #TODO: Refactor error logging, 2006/10/13 shino
-        @logger.error(error_message)
-        @logger.info{response.to_yaml}
-        #TODO: must create AP4R specific Exception class, 2006/10/12 shino
-        raise StandardError.new(error_message)
-      end
-
-      response
-    end
-
-    def dispatch_via_xmlrpc(message)
-      endpoint = message[:target_url]
-      client = XMLRPC::Client.new2(endpoint)
-      success, response = client.call2(message[:target_action], message.object)
-      raise response unless success
-      response
-    end
-
-    def dispatch_via_soap(message)
-      @logger.debug{message[:target_url]}
-      driver = SOAP::WSDLDriverFactory.new(message[:target_url]).create_rpc_driver
-      @logger.debug{driver}
-      driver.send(message[:target_action], message.object)
-    end
-
-    def stop_dispatchers
-      @logger.info{"stop_dispatchers #{@disps}"}
-      return unless @disps
-      @disps.list.each {|d| d[:dying] = true}
-      @disps.list.each {|d| d.join }
-    end
- 
-    def start_carriers
-      conf = @config.carriers
-      return unless conf
-
-      @logger.info{ "ready to start carrires with config #{conf.to_yaml}" }
-      @carriers = ThreadGroup.new
-      conf.each { |remote|
-        remote["threads"].times { |index|
-          Thread.fork(@carriers, remote, index){|group, remote, index|
-            # TODO: refactor structure, 2006/10/06 shino
-            group.add Thread.current
-            @logger.info{ "starting a carrier (index #{index}) for the queue manager #{remote['source_uri']}" }
-            uri = remote['source_uri']
-            until Thread.current[:dying]
-              begin
-                sleep 0.1
-                #TODO check :dying flag here and break, 2006/09/01 shino
-                #TODO cache DRbObject if necessary, 2006/09/01 shino
-                remote_qm = DRb::DRbObject.new_with_uri(uri)
-                queue_name = remote_qm.stale_queue @dispatch_targets
-                next unless queue_name
-
-                @logger.debug{ "stale queue name : #{queue_name}" }
-                q = ReliableMsg::Queue.new queue_name, :drb_uri => uri
-                q.get { |m|
-                  unless m
-                    @logger.debug{ "carrier strikes at the air (T_T)" }
-                    next
-                  end
-                  # @logger.debug{ "carrier gets a message\n#{m.to_yaml}" }
-
-                  # TODO: decide the better one, and delete another, 2006/09/01 shino
-                  # TODO: or switchable implementation in versions, 2006/10/16 shino
-
-                  #version 1: use thread fork so queue manager use a different tx
-                  # TODO probably should have a thread as an instance variable or in a thread local, 2006/09/01 shino
-                  #Thread.fork(m) {|m|
-                  #  local_queue = ReliableMsg::Queue.new queue_name
-                  #  local_queue.put m.object
-                  #}.join
-                  
-                  #version 2: store tx and set nil, and resotre tx after putting a message
-                  begin
-                    tx = Thread.current[ReliableMsg::Client::THREAD_CURRENT_TX]
-                    Thread.current[ReliableMsg::Client::THREAD_CURRENT_TX] = nil
-                    # @logger.debug{ "before tx: #{tx}" }
-                    ReliableMsg::Queue.new(queue_name).put(m.object)
-                  ensure
-                    Thread.current[ReliableMsg::Client::THREAD_CURRENT_TX] = tx
-                    # @logger.debug{ "after tx: #{Thread.current[ReliableMsg::Client::THREAD_CURRENT_TX]}" }
-                  end
-                }
-              rescue Exception => ex
-                @logger.warn "error in remote-get/local-put #{ex}\n#{ex.backtrace.join("\n\t")}\n"
-              end
-            end
-            @logger.info{"ends a carrier (index #{index}) for the queue manager #{remote['uri']}"}
-          }
-        }
-      }
-      @logger.info{"queue manager has forked all carriers"}
-    end
-
-    def stop_carriers
-      @logger.info{"stop_carriers #{@carriers}"}
-      return unless @carriers
-      @carriers.list.each{|d| d[:dying] = true}
-      @carriers.list.each{|d| d.join }
-    end
 
   end
 end

data/lib/ap4r/queue_manager_ext_debug.rb

@@ -1,12 +1,12 @@
 # Author:: Shunichi Shinohara
-# Copyright:: Copyright (c) 2006 Future System Consulting Corp.
+# Copyright:: Copyright (c) 2007 Future Architect Inc.
 # Licence:: MIT Licence
 
 require 'drb/drb'
 
-module ReliableMsg
+module ReliableMsg #:nodoc:
   class QueueManager
-    attr_reader :store, :transactions, :mutex, :config
+    attr_reader :store, :transactions, :mutex, :config, :dispatchers, :carriers
 
     # Accepts ruby code as a string, evaluates it on +self+,
     # and returns the result as a formatted string.
@@ -38,9 +38,16 @@ module ReliableMsg
     end
     alias e2i eval_to_inspect
 
+    # Checks queues are all "empty".
+    #
+    # "Empty" means no messages in transaction and
+    # all queues but <tt>$dlq</tt> are empty.
+    def no_active_message?
+      @transactions.size.zero? && @store.queues.all?{|(q, ms)| q == "$dlq" ||  ms.size.zero? }
+    end
   end
   
-  module MessageStore
+  module MessageStore #:nodoc:
     class Base
       include DRbUndumped
       attr_reader :mutex, :queues, :topics, :cache
@@ -50,9 +57,10 @@ module ReliableMsg
       def activate
         activate_original
         @mutex.extend DRbUndumped
-#        @queues.extend DRbUndumped
-#        @topics.extend DRbUndumped
-#        @cache.extend DRbUndumped
+        # TODO: queues/topics/cache should be DRbUndumped? 2007/06/06 by shino
+        # @queues.extend DRbUndumped
+        # @topics.extend DRbUndumped
+        # @cache.extend DRbUndumped
       end
 
     end