reliable-msg-agent 0.0.1 → 0.0.2

data/History.txt

@@ -1,4 +1,12 @@
 
+== 0.0.2 2011-03-19
+
+* resources/ was includ in gemfiles.
+* renamed workers to consumers.
+* refactored drb connect method.
+* changed Agent#call arguments 2 to 3.
+* added configuration examples to resources/
+
 == 0.0.1 2011-03-18
 
 * minimum feature

data/Rakefile

@@ -16,7 +16,7 @@ spec = Gem::Specification.new do |s|
   s.summary = "This(it) is agent daemon for reliable-msg queue"
   s.description = s.summary
   s.executables = Dir.glob("bin/**/*").map { |f| File.basename(f) }
-  s.files = %w(History.txt MIT-LICENSE README.rdoc Rakefile) + Dir.glob("{bin,ext,lib,spec}/**/*")
+  s.files = %w(History.txt MIT-LICENSE README.rdoc Rakefile) + Dir.glob("{bin,ext,lib,resources,spec}/**/*")
   s.require_path = "lib"
   s.bindir = "bin"
   s.author = "hamajyotan"

data/lib/reliable-msg/agent/agent.rb

@@ -8,6 +8,17 @@ module ReliableMsg::Agent #:nodoc:
       @logger = logger
     end
  
+    # The method of processing the message is defined.
+    #
+    # if the evaluation result is nil or false,
+    # it is considered that it failes.
+    #
+    # === Args
+    #
+    # +msg+     :: fetched message from reliable-msg queue.
+    # +conf+    :: consumer configurations.
+    # +options+ :: the options (it is still unused.)
+    #
     def call msg, options = {}
       raise AgentError, "#call(msg,options={}) not implemented!"
     end   

data/lib/reliable-msg/agent/consumers.rb

@@ -0,0 +1,114 @@
+
+require "rubygems"
+require "reliable-msg/agent"
+
+require "monitor"
+
+module ReliableMsg::Agent #:nodoc:
+  class Consumers
+
+    @@default_options = {
+      "source_uri" => "druby://localhost:6438",
+      "target"     => "queue.*",
+      "every"      => 1.0,
+      "threads"    => 1,
+    }.freeze
+
+    def initialize logger, options
+      @logger  = logger
+      @options = options
+      raise AgentError, "no configuration specified."  unless @options
+      @locker  = Monitor.new
+      @threads = nil
+    end
+
+    def start
+      @locker.synchronize {
+        raise AgentError, "workers already started." if alive?
+
+        @logger.info { "--- starting workers." }
+
+        @threads = []
+        @options.each { |opts|
+          conf = @@default_options.merge(opts || {})
+          conf["threads"].to_i.times {
+            @threads << Thread.fork(conf) { |c| consuming_loop(c) }
+          }
+        }
+      }
+    end
+
+    def stop
+      @locker.synchronize {
+        raise AgentError, "workers already stopped." unless alive?
+
+        @logger.info { "--- stopping workers." }
+
+        @threads.each { |t| t[:dying] = true }
+        @threads.each { |t| t.wakeup rescue nil } # for immediate stop
+        @threads.each { |t| t.join }
+        @threads = nil
+      }
+    end
+
+    def alive?; @locker.synchronize { !! @threads }; end
+
+    private
+
+    def consuming_loop conf
+      agent = Agent.new @logger
+      uri, every, target = conf["source_uri"], conf["every"], conf["target"]
+
+      until Thread.current[:dying]
+        begin
+          remote_qm = DRb::DRbObject.new_with_uri uri
+
+          queue_name = connect_qm(remote_qm, target, uri, every)
+          next unless queue_name
+
+          fetch(queue_name, uri) { |m|
+            raise AgentError, "agent proc failed." unless agent.call(m, conf)
+          }
+
+        rescue Exception => e
+          @logger.warn { "error in fetch-msg/agent-proc: #{e}\n#{e.backtrace.join("\n\t")}" }
+        end
+
+        sleep every
+      end
+    end
+
+    def connect_qm qm, target, uri, every
+      error_raised = false
+      begin
+        queue_name = qm.stale_queue target
+        @logger.warn { "Connect to #{uri} successfully at #{Time.now}" } if error_raised
+        return queue_name
+
+      rescue => e
+        @logger.warn { "Lost connection to #{uri} at #{Time.now} - #{e.message}" } unless error_raised
+        error_raised = true
+        sleep every
+        return nil if Thread.current[:dying]
+        retry
+      end
+    end
+
+    def fetch queue_name, source_uri
+      ReliableMsg::Queue.new(queue_name, :drb_uri => source_uri).get { |m|
+        begin
+          tx = Thread.current[ReliableMsg::Client::THREAD_CURRENT_TX]
+          Thread.current[ReliableMsg::Client::THREAD_CURRENT_TX] = nil
+
+          @logger.info { "message fetched - <#{m.id}>" }
+          yield m
+
+        ensure
+          Thread.current[ReliableMsg::Client::THREAD_CURRENT_TX] = tx
+        end
+      }
+    end
+
+  end
+end
+

data/lib/reliable-msg/agent/service.rb

@@ -10,7 +10,7 @@ module ReliableMsg::Agent #:nodoc:
       @logger  = logger
       @options = options
 
-      @workers = Workers.new @logger, @options
+      @workers = Consumers.new @logger, @options["consumers"]
       @locker = Monitor.new
     end
 
@@ -18,9 +18,9 @@ module ReliableMsg::Agent #:nodoc:
       @locker.synchronize {
         raise AgentError, "service already started." if alive?
 
-        @logger.info { "reliable-msg agent service starting..." }
+        @logger.info { "*** reliable-msg agent service starting..." }
         @workers.start
-        @logger.info { "reliable-msg agent service started." }
+        @logger.info { "*** reliable-msg agent service started." }
       }
     end
 
@@ -28,9 +28,9 @@ module ReliableMsg::Agent #:nodoc:
       @locker.synchronize {
         raise AgentError, "service already stopped." unless alive?
 
-        @logger.info { "reliable-msg agent service stopping..." }
+        @logger.info { "*** reliable-msg agent service stopping..." }
         @workers.stop
-        @logger.info { "reliable-msg agent service stopped." }
+        @logger.info { "*** reliable-msg agent service stopped." }
       }
     end
 

data/lib/reliable-msg/agent/version.rb

@@ -7,7 +7,7 @@ module ReliableMsg::Agent #:nodoc:
     unless defined? MAJOR
       MAJOR  = 0
       MINOR  = 0
-      TINY   = 1
+      TINY   = 2
       PRE    = nil
 
       STRING = [MAJOR, MINOR, TINY, PRE].compact.join(".")

data/lib/reliable-msg/agent.rb

@@ -4,11 +4,11 @@ require "reliable-msg"
 
 module ReliableMsg #:nodoc:
   module Agent #:nodoc:
-    autoload :Version   , "reliable-msg/agent/version"
+    autoload :Agent     , "reliable-msg/agent/agent"
     autoload :AgentError, "reliable-msg/agent/error"
+    autoload :Consumers , "reliable-msg/agent/consumers"
     autoload :Service   , "reliable-msg/agent/service"
-    autoload :Workers   , "reliable-msg/agent/workers"
-    autoload :Agent     , "reliable-msg/agent/agent"
+    autoload :Version   , "reliable-msg/agent/version"
   end
 end
 

data/resources/agent.conf

@@ -0,0 +1,13 @@
+---
+user: agent
+group: agent
+logger: " Proc.new { |file| l = Logger.new(file); l.level = Logger::DEBUG; l } "
+agent: /etc/reliable-msg-agent/agent.rb
+
+consumers:
+  -
+    source_uri: druby://localhost:6438
+    every: 1.0
+    target: queue.*
+    threads: 1
+

data/resources/agent.rb

@@ -0,0 +1,21 @@
+
+# this script is evaluated by the context of ReliableMsg::Agnet::Agent class.
+
+require "yaml"
+
+#
+# The method of processing the message is defined.
+#
+# if the evaluation result is nil or false,
+# it is considered that it failes.
+#
+# === Args
+#
+# +msg+     :: fetched message from reliable-msg queue.
+# +conf+    :: consumer configurations.
+# +options+ :: the options (it is still unused.)
+#
+def call msg, conf, options = {}
+  @logger.info { "message received\n#{msg.to_yaml}" }
+end
+

data/resources/examples/ap4r-dispatch-agent.conf

@@ -0,0 +1,26 @@
+---
+user: agent
+group: agent
+logger: " Proc.new { |file| l = Logger.new(file); l.level = Logger::DEBUG; l } "
+agent: /etc/reliable-msg-agent/agent.rb
+
+consumers:
+  -
+    source_uri: druby://localhost:6438
+    every: 1.0
+    target: queue.foo
+    threads: 3
+    modify_rules:
+      url: " Proc.new { |url| url.port = 3001 + rand(3); url } "
+    http:
+      timeout: 30
+  -
+    source_uri: druby://localhost:6438
+    every: 1.0
+    target: queue.bar
+    threads: 3
+    modify_rules:
+      url: " Proc.new { |url| url.port = 4001 + rand(3); url } "
+    http:
+      timeout: 60
+

data/resources/examples/ap4r-dispatch-agent.rb

@@ -0,0 +1,32 @@
+
+# this script is evaluated by the context of ReliableMsg::Agnet::Agent class.
+
+require "yaml"
+require "ap4r"
+
+#
+# The method of processing the message is defined.
+#
+# if the evaluation result is nil or false,
+# it is considered that it failes.
+#
+# === Args
+#
+# +msg+     :: fetched message from reliable-msg queue.
+# +conf+    :: consumer configurations.
+# +options+ :: the options (it is still unused.)
+#
+def call msg, conf, options = {}
+
+  # The following codes use the mechanism of sending the message by ap4r.
+  dispatcher = Ap4r::Dispatchers.new nil, [], @logger
+
+  @logger.debug { "dispatcher get message\n#{msg.to_yaml}" }
+  response = dispatcher.send(:get_dispather_instance,
+                             msg.headers[:dispatch_mode],
+                             msg,
+                             conf).call
+  @logger.debug { "dispatcher get response\n#{response.to_yaml}" }
+
+end
+

data/resources/examples/log-level-change-agent.conf

@@ -0,0 +1,13 @@
+---
+user: agent
+group: agent
+logger: " Proc.new { |file| l = Logger.new(file); l.level = Logger::WARN; l } "
+agent: /etc/reliable-msg-agent/agent.rb
+
+consumers:
+  -
+    source_uri: druby://localhost:6438
+    every: 1.0
+    target: queue.*
+    threads: 1
+

data/resources/examples/log-rotate-daily-agent.conf

@@ -0,0 +1,13 @@
+---
+user: agent
+group: agent
+logger: " Proc.new { |file| Logger.new(file, 'daily') } "
+agent: /etc/reliable-msg-agent/agent.rb
+
+consumers:
+  -
+    source_uri: druby://localhost:6438
+    every: 1.0
+    target: queue.*
+    threads: 1
+

data/resources/examples/log-rotate-size-agent.conf

@@ -0,0 +1,13 @@
+---
+user: agent
+group: agent
+logger: " Proc.new { |file| Logger.new(file, 7, 10*1024*1024) } "
+agent: /etc/reliable-msg-agent/agent.rb
+
+consumers:
+  -
+    source_uri: druby://localhost:6438
+    every: 1.0
+    target: queue.*
+    threads: 1
+

data/resources/examples/simple-agent.rb

@@ -0,0 +1,21 @@
+
+# this script is evaluated by the context of ReliableMsg::Agnet::Agent class.
+
+require "yaml"
+
+#
+# The method of processing the message is defined.
+#
+# if the evaluation result is nil or false,
+# it is considered that it failes.
+#
+# === Args
+#
+# +msg+     :: fetched message from reliable-msg queue.
+# +conf+    :: consumer configurations.
+# +options+ :: the options (it is still unused.)
+#
+def call msg, conf, options = {}
+  @logger.info { "message received\n#{msg.to_yaml}" }
+end
+

data/resources/reliable-msg-agentd

@@ -0,0 +1,41 @@
+#!/bin/sh
+# reliable-msg-agent start/stop script.
+# chkconfig: - 85 15
+# description: relilable-msg-agent start/stop script.
+. /etc/rc.d/init.d/functions
+
+prog=reliable-msg-agentd
+
+base=reliable-msg-agent
+cnf=/etc/reliable-msg-agent/agent.conf
+pid=/var/reliable-msg-agent/agent.pid
+log=/var/reliable-msg-agent/agent.log
+
+start() {
+  echo $"Start ReliableMsg-Agent ..."
+  $base start -c $cnf -p $pid -l $log -d
+}
+
+stop() {
+  echo $"Stop ReliableMsg-Agent ..."
+  $base stop -p $pid
+}
+
+case "$1" in
+  start)
+    start
+    ;;
+  stop)
+    stop
+    ;;
+  restart)
+    stop
+    start
+    ;;
+  status)
+    status -p $pid $prog
+    ;;
+  *)
+    echo $"Usage: $prog {start|stop|restart|status}"
+esac
+

data/spec/consumers_spec.rb

@@ -0,0 +1,50 @@
+
+require File.dirname(__FILE__) + "/spec_helper.rb"
+
+require "logger"
+
+describe ReliableMsg::Agent::Consumers do
+  before do
+    logger = Logger.new(nil)
+    conf = [
+      {
+        "source_uri" => "druby://localhost:6438",
+        "target"     => "queue.*",
+        "every"      => 1.0,
+        "threads"    => 1,
+        "modify_rules" => {
+          "url" => "Proc.new { |url| url.host = '127.0.0.1'; url.port = 80; url }",
+        },
+        "http" => {
+          "timeout" => 60,
+        }
+      },
+    ]
+    @w = ReliableMsg::Agent::Consumers.new logger, conf
+  end
+
+  it "should be able to #start" do
+    @w.start
+  end
+
+  it "should not be alive" do
+    @w.alive?.should_not be_true
+  end
+
+  describe "When started" do
+    before do
+      @w.start
+    end
+
+    it "should be alive" do
+      @w.alive?.should be_true
+    end
+
+  end
+
+  after do
+    @w.stop rescue nil
+    @w = nil
+  end
+end
+

data/spec/service_spec.rb

@@ -5,8 +5,18 @@ require "logger"
 
 describe ReliableMsg::Agent::Service do
   before do
-    l = Logger.new(nil)
-    @s = ReliableMsg::Agent::Service.new l
+    logger = Logger.new(nil)
+    conf = {
+      "consumers" => [
+        {
+          "source_uri" => "druby://localhost:6438",
+          "target"     => "queue.*",
+          "every"      => 1.0,
+          "threads"    => 1,
+        },
+      ],
+    }
+    @s = ReliableMsg::Agent::Service.new logger, conf
   end
 
   it "should be able to #start" do

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: reliable-msg-agent
 version: !ruby/object:Gem::Version 
-  hash: 29
+  hash: 27
   prerelease: false
   segments: 
   - 0
   - 0
-  - 1
-  version: 0.0.1
+  - 2
+  version: 0.0.2
 platform: ruby
 authors: 
 - hamajyotan
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-03-18 00:00:00 +09:00
+date: 2011-03-19 00:00:00 +09:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -47,15 +47,24 @@ files:
 - Rakefile
 - bin/reliable-msg-agent
 - lib/reliable-msg/agent/error.rb
-- lib/reliable-msg/agent/workers.rb
 - lib/reliable-msg/agent/version.rb
 - lib/reliable-msg/agent/service.rb
 - lib/reliable-msg/agent/agent.rb
+- lib/reliable-msg/agent/consumers.rb
 - lib/reliable-msg/agent/scripts/script_runner.rb
 - lib/reliable-msg/agent.rb
+- resources/agent.conf
+- resources/reliable-msg-agentd
+- resources/examples/simple-agent.rb
+- resources/examples/log-rotate-size-agent.conf
+- resources/examples/log-level-change-agent.conf
+- resources/examples/log-rotate-daily-agent.conf
+- resources/examples/ap4r-dispatch-agent.rb
+- resources/examples/ap4r-dispatch-agent.conf
+- resources/agent.rb
 - spec/spec_helper.rb
 - spec/service_spec.rb
-- spec/workers_spec.rb
+- spec/consumers_spec.rb
 has_rdoc: true
 homepage: https://github.com/hamajyotan/reliable-msg-agent
 licenses: []

data/lib/reliable-msg/agent/workers.rb

@@ -1,90 +0,0 @@
-
-require "rubygems"
-require "reliable-msg/agent"
-
-require "monitor"
-
-module ReliableMsg::Agent #:nodoc:
-  class Workers
-
-    @@default_options = {
-      :uri     => "druby://localhost:6438",
-      :every   => 1.0,
-      :target  => "queue.*",
-      :threads => 1,
-    }.freeze
-
-    def initialize logger, options = {}
-      @logger  = logger
-      @options = options
-      @locker  = Monitor.new
-      @threads = nil
-    end
-
-    def start
-      @locker.synchronize {
-        raise AgentError, "workers already started." if alive?
-
-        @logger.info { "starting workers." }
-
-        threads = (t = @options[:threads].to_i) <= 0 ? 1 : t
-        @threads = (1..threads).inject([]) { |t, i|
-          t << Thread.fork { worker_loop }
-        }
-      }
-    end
-
-    def stop
-      @locker.synchronize {
-        raise AgentError, "workers already stopped." unless alive?
-
-        @logger.info { "stopping workers." }
-
-        @threads.each { |t| t[:dying] = true }
-        @threads.each { |t| t.wakeup rescue nil }
-        @threads.each { |t| t.join }
-        @threads = nil
-      }
-    end
-
-    def alive?
-      @locker.synchronize { !! @threads }
-    end
-
-    private
-
-    def worker_loop
-      agent  = Agent.new @logger
-      uri    = @options[:uri]    || @@default_options[:uri]
-      every  = @options[:every]  || @@default_options[:every]
-      target = @options[:target] || @@default_options[:target]
-
-      until Thread.current[:dying]
-        begin
-          sleep every
-          
-          remote_qm = DRb::DRbObject.new_with_uri uri
-          queue_name = remote_qm.stale_queue target
-          next unless queue_name
-  
-          q = ReliableMsg::Queue.new queue_name, :drb_uri => uri
-          q.get { |m|
-            begin
-              tx = Thread.current[ReliableMsg::Client::THREAD_CURRENT_TX]
-              Thread.current[ReliableMsg::Client::THREAD_CURRENT_TX] = nil
-
-              @logger.info { "message received - <#{m.id}>" }
-              raise AgentError, "agent proc failed."  unless agent.call(m.dup)
-            ensure
-              Thread.current[ReliableMsg::Client::THREAD_CURRENT_TX] = tx
-            end
-          }
-        rescue Exception => e
-          @logger.warn { "error in fetch-msg/agent-proc: #{e}\n#{e.backtrace.join("\n\t")}" }
-        end
-      end
-    end
-
-  end
-end
-

data/spec/workers_spec.rb

@@ -1,36 +0,0 @@
-
-require File.dirname(__FILE__) + "/spec_helper.rb"
-
-require "logger"
-
-describe ReliableMsg::Agent::Workers do
-  before do
-    l = Logger.new(nil)
-    @w = ReliableMsg::Agent::Workers.new l
-  end
-
-  it "should be able to #start" do
-    @w.start
-  end
-
-  it "should not be alive" do
-    @w.alive?.should_not be_true
-  end
-
-  describe "When started" do
-    before do
-      @w.start
-    end
-
-    it "should be alive" do
-      @w.alive?.should be_true
-    end
-
-  end
-
-  after do
-    @w.stop rescue nil
-    @w = nil
-  end
-end
-