isono 0.2.3 → 0.2.6

data/Rakefile

@@ -24,7 +24,7 @@ task :gem do
     s.bindir='bin'
     s.executables = %w(cli)
     
-    s.add_dependency "amqp", "0.7.0"
+    s.add_dependency "amqp", "0.7.4"
     s.add_dependency "eventmachine", "1.0.0.beta.3"
     s.add_dependency "statemachine", ">= 1.0.0"
     s.add_dependency "log4r"

data/isono.gemspec

@@ -2,35 +2,33 @@
 
 Gem::Specification.new do |s|
   s.name = %q{isono}
-  s.version = "0.2.3"
+  s.version = "0.2.6"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
-  s.authors = ["axsh Ltd.", "Masahiro Fujiwara"]
-  s.date = %q{2011-06-29}
-  s.default_executable = %q{cli}
-  s.email = ["dev@axsh.net", "m-fujiwara@axsh.net"]
-  s.executables = ["cli"]
-  s.files = [".gitignore", "LICENSE", "NOTICE", "Rakefile", "bin/cli", "isono.gemspec", "lib/ext/shellwords.rb", "lib/isono.rb", "lib/isono/amqp_client.rb", "lib/isono/daemonize.rb", "lib/isono/event_delegate_context.rb", "lib/isono/event_observable.rb", "lib/isono/logger.rb", "lib/isono/manifest.rb", "lib/isono/messaging_client.rb", "lib/isono/models/event_log.rb", "lib/isono/models/job_state.rb", "lib/isono/models/node_state.rb", "lib/isono/models/resource_instance.rb", "lib/isono/node.rb", "lib/isono/node_modules/base.rb", "lib/isono/node_modules/data_store.rb", "lib/isono/node_modules/event_channel.rb", "lib/isono/node_modules/event_logger.rb", "lib/isono/node_modules/job_channel.rb", "lib/isono/node_modules/job_collector.rb", "lib/isono/node_modules/job_worker.rb", "lib/isono/node_modules/node_collector.rb", "lib/isono/node_modules/node_heartbeat.rb", "lib/isono/node_modules/rpc_channel.rb", "lib/isono/rack.rb", "lib/isono/rack/builder.rb", "lib/isono/rack/data_store.rb", "lib/isono/rack/job.rb", "lib/isono/rack/map.rb", "lib/isono/rack/object_method.rb", "lib/isono/rack/proc.rb", "lib/isono/rack/thread_pass.rb", "lib/isono/resource_manifest.rb", "lib/isono/runner/base.rb", "lib/isono/runner/cli.rb", "lib/isono/runner/rpc_server.rb", "lib/isono/serializer.rb", "lib/isono/thread_pool.rb", "lib/isono/util.rb", "lib/isono/version.rb", "spec/amqp_client_spec.rb", "spec/event_observable_spec.rb", "spec/file_channel_spec.rb", "spec/job_channel_spec.rb", "spec/logger_spec.rb", "spec/manifest_spec.rb", "spec/node_spec.rb", "spec/resource_loader_spec.rb", "spec/rpc_channel_spec.rb", "spec/spec_helper.rb", "spec/thread_pool_spec.rb", "spec/util_spec.rb", "tasks/load_resource_manifest.rake"]
+  s.authors = [%q{axsh Ltd.}, %q{Masahiro Fujiwara}]
+  s.date = %q{2011-08-23}
+  s.email = [%q{dev@axsh.net}, %q{m-fujiwara@axsh.net}]
+  s.executables = [%q{cli}]
+  s.files = [%q{.gitignore}, %q{LICENSE}, %q{NOTICE}, %q{Rakefile}, %q{bin/cli}, %q{isono.gemspec}, %q{lib/ext/shellwords.rb}, %q{lib/isono.rb}, %q{lib/isono/amqp_client.rb}, %q{lib/isono/daemonize.rb}, %q{lib/isono/event_delegate_context.rb}, %q{lib/isono/event_observable.rb}, %q{lib/isono/logger.rb}, %q{lib/isono/manifest.rb}, %q{lib/isono/messaging_client.rb}, %q{lib/isono/models/event_log.rb}, %q{lib/isono/models/job_state.rb}, %q{lib/isono/models/node_state.rb}, %q{lib/isono/models/resource_instance.rb}, %q{lib/isono/node.rb}, %q{lib/isono/node_modules/base.rb}, %q{lib/isono/node_modules/data_store.rb}, %q{lib/isono/node_modules/event_channel.rb}, %q{lib/isono/node_modules/event_logger.rb}, %q{lib/isono/node_modules/job_channel.rb}, %q{lib/isono/node_modules/job_collector.rb}, %q{lib/isono/node_modules/job_worker.rb}, %q{lib/isono/node_modules/node_collector.rb}, %q{lib/isono/node_modules/node_heartbeat.rb}, %q{lib/isono/node_modules/rpc_channel.rb}, %q{lib/isono/rack.rb}, %q{lib/isono/rack/builder.rb}, %q{lib/isono/rack/data_store.rb}, %q{lib/isono/rack/job.rb}, %q{lib/isono/rack/map.rb}, %q{lib/isono/rack/object_method.rb}, %q{lib/isono/rack/proc.rb}, %q{lib/isono/rack/thread_pass.rb}, %q{lib/isono/resource_manifest.rb}, %q{lib/isono/runner/base.rb}, %q{lib/isono/runner/cli.rb}, %q{lib/isono/runner/rpc_server.rb}, %q{lib/isono/serializer.rb}, %q{lib/isono/thread_pool.rb}, %q{lib/isono/util.rb}, %q{lib/isono/version.rb}, %q{spec/amqp_client_spec.rb}, %q{spec/event_observable_spec.rb}, %q{spec/file_channel_spec.rb}, %q{spec/job_channel_spec.rb}, %q{spec/logger_spec.rb}, %q{spec/manifest_spec.rb}, %q{spec/node_spec.rb}, %q{spec/resource_loader_spec.rb}, %q{spec/rpc_channel_spec.rb}, %q{spec/spec_helper.rb}, %q{spec/thread_pool_spec.rb}, %q{spec/util_spec.rb}, %q{tasks/load_resource_manifest.rake}]
   s.homepage = %q{http://github.com/axsh/isono}
-  s.require_paths = ["lib"]
+  s.require_paths = [%q{lib}]
   s.required_ruby_version = Gem::Requirement.new(">= 1.8.7")
   s.rubyforge_project = %q{isono}
-  s.rubygems_version = %q{1.3.7}
+  s.rubygems_version = %q{1.8.6}
   s.summary = %q{Messaging and agent fabric}
 
   if s.respond_to? :specification_version then
-    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
     s.specification_version = 3
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency(%q<amqp>, ["= 0.7.0"])
+      s.add_runtime_dependency(%q<amqp>, ["= 0.7.4"])
       s.add_runtime_dependency(%q<eventmachine>, ["= 1.0.0.beta.3"])
       s.add_runtime_dependency(%q<statemachine>, [">= 1.0.0"])
       s.add_runtime_dependency(%q<log4r>, [">= 0"])
       s.add_development_dependency(%q<bacon>, [">= 0"])
       s.add_development_dependency(%q<rake>, [">= 0"])
     else
-      s.add_dependency(%q<amqp>, ["= 0.7.0"])
+      s.add_dependency(%q<amqp>, ["= 0.7.4"])
       s.add_dependency(%q<eventmachine>, ["= 1.0.0.beta.3"])
       s.add_dependency(%q<statemachine>, [">= 1.0.0"])
       s.add_dependency(%q<log4r>, [">= 0"])
@@ -38,7 +36,7 @@ Gem::Specification.new do |s|
       s.add_dependency(%q<rake>, [">= 0"])
     end
   else
-    s.add_dependency(%q<amqp>, ["= 0.7.0"])
+    s.add_dependency(%q<amqp>, ["= 0.7.4"])
     s.add_dependency(%q<eventmachine>, ["= 1.0.0.beta.3"])
     s.add_dependency(%q<statemachine>, [">= 1.0.0"])
     s.add_dependency(%q<log4r>, [">= 0"])

data/lib/isono/amqp_client.rb

@@ -4,7 +4,6 @@ require 'thread'
 
 require 'eventmachine'
 require 'amqp'
-require 'mq'
 
 require 'uri/generic'
 
@@ -155,7 +154,7 @@ module Isono
     # @note Do not have to close by user. Channel close is performed
     #       as part of connection close.
     def create_channel
-      MQ.new(@amqp_client)
+      AMQP::Channel.new(@amqp_client)
     end
 
     # Publish a message to the designated exchange.

data/lib/isono/node_modules/job_worker.rb

@@ -1,7 +1,5 @@
 # -*- coding: utf-8 -*-
 
-require 'statemachine'
-
 module Isono
   module NodeModules
     class JobWorker < Base
@@ -55,14 +53,14 @@ module Isono
         @thread_pool.pass {
           begin
             Thread.current[JOB_CTX_KEY]=job
-            job.stm.on_start
+            job.process_event(:on_start)
             rpc.request('job-collector', 'update', job.to_hash) { |req|
               req.oneshot = true
             }
             job.run_cb.call
-            job.stm.on_done
+            job.process_event(:on_done)
           rescue Exception => e
-            job.stm.on_fail(e)
+            job.process_event(:on_fail, e)
             if job.fail_cb
               job.fail_cb.arity == 1 ? job.fail_cb.call(e) : job.fail_cb.call
             end
@@ -109,35 +107,46 @@ module Isono
 
       class JobContext < OpenStruct
         include Logger
-        attr_reader :stm
         attr_accessor :run_cb, :fail_cb
+        attr_reader :state
         
         def initialize()
           super({:job_id=>Util.gen_id,
                   :parent_job_id=>nil,
                   :started_at=>nil,
                   :finished_at=>nil,
+                  :finish_status=>nil,
                 })
 
           @run_cb=proc{}
           @fail_cb=nil
 
-          @stm = Statemachine.build {
-            startstate :init
-            trans :init,   :on_start,  :running, :on_start
-            trans :running, :on_done, :done, :on_done
-            trans :running, :on_fail, :failed, :on_fail
-            trans :init,   :on_fail, :failed, :on_fail
-          }
-          @stm.context = self
+          @state = :init
         end
 
-        def state
-          stm.state
+        def process_event(ev, *args)
+          case [ev,@state]
+          when [:on_start, :init]
+            @state = :running
+            self.started_at = Time.now
+            logger.info("Job start #{job_id}")
+          when [:on_done, :running]
+            @state = :done
+            self.finished_at = Time.now
+            logger.info("Job complete #{job_id}: #{elapsed_time} sec")
+          when [:on_fail, :running]
+            @state = :failed
+            on_fail(args[0])
+          when [:on_fail, :init]
+            @state = :failed
+            on_fail(args[0])
+          else
+            raise "Unknown state transition: #{ev}, #{@state}"
+          end
         end
         
         def to_hash
-          @table.dup.merge({:state=>@stm.state})
+          @table.dup.merge({:state=>@state})
         end
 
         def elapsed_time
@@ -149,16 +158,6 @@ module Isono
         end
 
         private
-        def on_start
-          self.started_at = Time.now
-          logger.info("Job start #{job_id}")
-        end
-
-        def on_done
-          self.finished_at = Time.now
-          logger.info("Job complete #{job_id}: #{elapsed_time} sec")
-        end
-
         def on_fail(e)
           self.finished_at = Time.now
           logger.error("Job failed #{job_id}: #{e}")

data/lib/isono/node_modules/rpc_channel.rb

@@ -1,7 +1,6 @@
 # -*- coding: utf-8 -*-
 
 require 'thread'
-require 'statemachine'
 require 'ostruct'
 
 module Isono
@@ -16,13 +15,27 @@ module Isono
       config_section do
         desc "default timeout duration in second until receive response."
         timeout_sec (60*3).to_f
+        desc "Prefetch size for receiver queue channel"
+        receiver_prefetch_size 10
       end
 
       initialize_hook do
         @active_requests = {}
+        @stats = {
+          :total_request_count=>0,
+          :total_response_count=>0,
+          :peak_wait_response_size=>0,
+          :send_request_per_sec=>0,
+          :receive_response_per_sec=>0,
+          :per_sec_last_request_count=>0,
+          :per_sec_last_response_count=>0,
+        }
         @endpoints = {}
         @amq = node.create_channel
-        @amq.queue("command-recv.#{manifest.node_id}", {:exclusive=>true}).subscribe { |header, data|
+        @amq.prefetch(config_section.receiver_prefetch_size.to_i)
+        @amq.queue("command-recv.#{manifest.node_id}", {:exclusive=>true}).subscribe(:ack=>true) { |header, data|
+          header.ack
+          @stats[:total_response_count] += 1
           req = @active_requests[header.message_id]
           if req
             data = Serializer.instance.unmarshal(data)
@@ -48,6 +61,26 @@ module Isono
           end
         }
 
+        # Cleanup timed out requests
+        EventMachine.add_periodic_timer(10) {
+          tnow = Time.now
+          @active_requests.each { |k,req|
+            next unless req.state == :waiting
+            if req.timeout_sec.to_f > 0.0 && req.timeout_sec.to_f > (tnow - req.sent_at)
+              @active_requests.delete(k)
+              req.error_cb.call(:timeout) if req.error_cb
+            end
+          }
+        }
+
+        # Update stats
+        EventMachine.add_periodic_timer(5) {
+          @stats[:send_request_per_sec] = (@stats[:total_request_count] - @stats[:per_sec_last_request_count]) / 5
+          @stats[:per_sec_last_request_count] = @stats[:total_request_count]
+          @stats[:receive_response_per_sec] = (@stats[:total_response_count] - @stats[:per_sec_last_response_count]) / 5
+          @stats[:per_sec_last_response_count] = @stats[:total_response_count]
+        }
+
         # RPC endpoint for statistics info of this node.
         myinstance.register_endpoint("rpc-stats.#{manifest.node_id}", proc { |req, res|
                                        case req.command
@@ -169,7 +202,7 @@ module Isono
             end
 
             def response_exchange
-              self.direct('')
+              self.default_exchange
             end
           }
           ch.prefetch(opts[:prefetch].to_i) if opts[:prefetch].to_i > 0
@@ -220,27 +253,23 @@ module Isono
           # set default timeout if no one updated the initial value.
           req.timeout_sec = config_section.timeout_sec
         end
-        
-        if req.timeout_sec > 0.0
-          # register the timeout hook.
-          req.timer = EventMachine::Timer.new(req.timeout_sec) {
-            @active_requests.delete req.ticket
-            req.error_cb.call(:timeout) if req.error_cb
-          }
-        end
-        
+                
         req.process_event(:on_ready)
         
         EventMachine.schedule {
+          @stats[:total_request_count] += 1
+          if @stats[:peak_wait_response_size] < @active_requests.size
+            @stats[:peak_wait_response_size] = @active_requests.size
+          end
           if !req.oneshot
             @active_requests[req.ticket] = req
           end
 
-          amq.direct('').publish(Serializer.instance.marshal(req.request_hash),
-                       {:message_id => req.ticket,
-                         :key => endpoint_queue_name(req.endpoint),
-                         :reply_to=>"command-recv.#{manifest.node_id}"}
-                       )
+          amq.default_exchange.publish(Serializer.instance.marshal(req.request_hash),
+                                       {:message_id => req.ticket,
+                                         :key => endpoint_queue_name(req.endpoint),
+                                         :reply_to=>"command-recv.#{manifest.node_id}"}
+                                       )
           req.process_event(:on_sent)
         }
       end
@@ -306,7 +335,7 @@ module Isono
       class RequestContext < OpenStruct
         # They are not to be appeared in @table so that won't be inspect().
         attr_reader :error_cb, :success_cb, :progress_cb
-        attr_accessor :timer
+        attr_reader :state
 
         def initialize(endpoint, command, args)
           super({:request=>{
@@ -314,8 +343,6 @@ module Isono
                     :command => command,
                     :args => args
                   },
-                  :endpoint=> endpoint,
-                  :command => command,
                   :ticket => Util.gen_id,
                   :timeout_sec => -1.0,
                   :oneshot => false,
@@ -327,36 +354,40 @@ module Isono
           @success_cb = nil
           @progress_cb = nil
           @error_cb = nil
-          @timer = nil
-
-          @stm = Statemachine.build {
-            trans :init, :on_ready, :ready
-            trans :ready, :on_sent, :waiting, proc {
-              self.sent_at=Time.now
-              # freeze request hash not to be modified after sending.
-              self.request.freeze
-            }
-            trans :waiting, :on_received, :waiting
-            trans :waiting, :on_error, :done, proc {
-              self.completed_at=Time.now
-              @timer.cancel if @timer
-              self.complete_status = :fail
-            }
-            trans :waiting, :on_success, :done, proc {
-              self.completed_at=Time.now
-              @timer.cancel if @timer
-              self.complete_status = :success
-            }
-          }
-          @stm.context = self
+
+          @state = :init
         end
 
-        def state
-          @stm.state
+        def endpoint
+          self.request[:endpoint]
+        end
+
+        def command
+          self.request[:command]
         end
 
         def process_event(ev, *args)
-          @stm.process_event(ev, *args)
+          case [ev, @state]
+          when [:on_ready, :init]
+            @state = :ready
+          when [:on_sent, :ready]
+            @state = :waiting
+            self.sent_at=Time.now
+            # freeze request hash not to be modified after sending.
+            self.request.freeze
+          when [:on_received, :waiting]
+            @state = :waiting
+          when [:on_success, :waiting]
+            @state = :done
+            self.completed_at=Time.now
+            self.complete_status = :success
+          when [:on_error, :waiting]
+            @state = :done
+            self.completed_at=Time.now
+            self.complete_status = :fail
+          else
+            raise "Unknown state transition: #{ev}, #{@state}"
+          end
         end
 
         def elapsed_time

data/lib/isono/runner/rpc_server.rb

@@ -97,7 +97,6 @@ module Isono
             app = app_builder.call(@builders[:rpc])
             if app
               NodeModules::RpcChannel.new(node).register_endpoint(endpoint, Rack.build do
-                                                                    use Rack::ThreadPass
                                                                     run app
                                                                   end
                                                                   )

data/lib/isono/util.rb

@@ -33,7 +33,7 @@ module Isono
     module_function :snake_case
     
     def gen_id(str=nil)
-      Digest::SHA1.hexdigest( (str.nil? ? rand.to_s : str) )
+      Digest::SHA1.hexdigest( (str.nil? ? ::Kernel.rand.to_s : str) )
     end
     module_function :gen_id
 

data/lib/isono/version.rb

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
 
 module Isono
-  VERSION='0.2.3'
+  VERSION='0.2.6'
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: isono
 version: !ruby/object:Gem::Version 
-  hash: 17
-  prerelease: false
+  hash: 27
+  prerelease: 
   segments: 
   - 0
   - 2
-  - 3
-  version: 0.2.3
+  - 6
+  version: 0.2.6
 platform: ruby
 authors: 
 - axsh Ltd.
@@ -16,8 +16,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-06-29 00:00:00 +09:00
-default_executable: cli
+date: 2011-08-23 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: amqp
@@ -27,12 +26,12 @@ dependencies:
     requirements: 
     - - "="
       - !ruby/object:Gem::Version 
-        hash: 3
+        hash: 11
         segments: 
         - 0
         - 7
-        - 0
-        version: 0.7.0
+        - 4
+        version: 0.7.4
   type: :runtime
   version_requirements: *id001
 - !ruby/object:Gem::Dependency 
@@ -181,7 +180,6 @@ files:
 - spec/thread_pool_spec.rb
 - spec/util_spec.rb
 - tasks/load_resource_manifest.rake
-has_rdoc: true
 homepage: http://github.com/axsh/isono
 licenses: []
 
@@ -213,7 +211,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: isono
-rubygems_version: 1.3.7
+rubygems_version: 1.8.6
 signing_key: 
 specification_version: 3
 summary: Messaging and agent fabric