isono 0.1.0 → 0.2.0

data/lib/isono/node_modules/base.rb

@@ -7,6 +7,7 @@ module Isono
       attr_reader :node
 
       def initialize(node)
+        raise ArgumentError unless node.is_a?(Node)
         @node = node
         
         raise "Module initializer_hook is not run yet" if self.value_object.nil?
@@ -18,10 +19,6 @@ module Isono
         node.manifest
       end
 
-      def amq
-        node.amq
-      end
-
       def value_object
         node.value_objects[self.class]
       end
@@ -34,19 +31,37 @@ module Isono
       
       module ClassMethods
         def initialize_hook(&blk)
-          @initialize_hook = blk if blk
-          @initialize_hook
+          @node_hooks[:initialize] = blk
         end
         
         def terminate_hook(&blk)
-          @terminate_hook = blk if blk
-          @terminate_hook
+          @node_hooks[:terminate] = blk
+        end
+
+        def before_connect_hook(&blk)
+          @node_hooks[:before_connect] = blk
+        end
+
+        def after_connect_hook(&blk)
+          @node_hooks[:after_connect] = blk
+        end
+        
+        def before_close_hook(&blk)
+          @node_hooks[:before_close] = blk
+        end
+
+        def after_close_hook(&blk)
+          @node_hooks[:after_close] = blk
         end
 
         def config_section(name=nil, &blk)
           @config_section_name = name unless name.nil? 
           @config_section_builder = blk
-        end        
+        end
+
+        def node_hooks
+          @node_hooks
+        end
       end
       
       protected
@@ -57,6 +72,7 @@ module Isono
           # set the default config section name from its class name.
           # can be overwritten later.
           @config_section_name = Util.snake_case(self.to_s.split('::').last)
+          @node_hooks = {}
         }
       end
       

data/lib/isono/node_modules/event_channel.rb

@@ -20,7 +20,18 @@ module Isono
       AMQP_EXCHANGE='isono.event'
       
       initialize_hook do
-        amq.topic(AMQP_EXCHANGE, {:auto_delete=>false})
+        @amq = node.create_channel
+        @amq.instance_eval {
+          def event_exchange
+            self.topic(AMQP_EXCHANGE, {:auto_delete=>false})
+          end
+        }
+
+        # create the exchange
+        @amq.event_exchange
+      end
+
+      terminate_hook do
       end
 
       # @example
@@ -37,14 +48,14 @@ module Isono
         }
         
         EventMachine.schedule {
-          amq.topic(AMQP_EXCHANGE).publish(Serializer.instance.marshal(body),
-                                           {:key=>"#{evname}.#{opts[:sender]}"}
-                                           )
+          @amq.event_exchange.publish(Serializer.instance.marshal(body),
+                                            {:key=>"#{evname}.#{opts[:sender]}"}
+                                            )
         }
       end
       
-      def subscribe(evname, sender, receiver_id=node.node_id, &blk)
-        amq.queue("#{evname}-#{receiver_id}", {:exclusive=>true}).bind(
+      def subscribe(evname, sender='#', receiver_id=node.node_id, &blk)
+        @amq.queue("#{evname}-#{receiver_id}", {:exclusive=>true}).bind(
                      AMQP_EXCHANGE, :key=>"#{evname}.#{sender}"
                      ).subscribe { |data|
           data = Serializer.instance.unmarshal(data)
@@ -62,7 +73,7 @@ module Isono
       
       def unsubscribe(evname, receiver_id=node.node_id)
         EventMachine.schedule {
-          q = amq.queue("#{evname}-#{receiver_id}")
+          q = @amq.queue("#{evname}-#{receiver_id}")
           q.unsubscribe
         }
       end

data/lib/isono/node_modules/job_channel.rb

@@ -4,7 +4,14 @@ module Isono
   module NodeModules
     class JobChannel < Base
 
-      # Send a new job request to the endpoint and get back job ID.
+      config_section do |c|
+        desc "default timeout duration in second"
+        c.timeout_sec = (60*60*24).to_f
+        desc "default job concurrency"
+        c.concurrency = 1
+      end
+
+      # Send a new job request to the endpoint.
       #
       # @param [String] endpoint endpoint name created by JobChannel#register_endpoint()
       # @param [String] command command name in the endpoint.
@@ -13,24 +20,14 @@ module Isono
       #   method will stop the current thread if it does not exist.
       # @return [Rack::Request,String] Request object if the block is given.
       #
-      # @example call job endpoint 'endpoint1' and receive 
-      #   submit('endpoint1', 'command1', 1, 2, 3) #=> Job ID.
+      # @example call job endpoint 'endpoint1'.
+      #  puts  submit('endpoint1', 'command1', 1, 2, 3) # => show Job ID
       def submit(endpoint, command, *args, &blk)
-        cur_job_ctx = Thread.current[JobWorker::JOB_CTX_KEY]
-        req = rpc.request("job.#{endpoint}", command, *args) { |req|
-          req.request[:job_request_type]=:submit
-
-          # A job is working on this current thread if cur_job_ctx is
-          # not nil. Let the new job know the current job ID
-          # as its parent job ID.
-          if cur_job_ctx
-            req.request[:parent_job_id] = cur_job_ctx.job_id
-          end
-
-          blk ? blk.call(req) : req.synchronize
+        req = run(endpoint, command, *args) { |req|
+          blk.call(req) if blk
         }
 
-        blk ? req : req.wait
+        req.request[:job_id]
       end
 
       # Send a new job request and wait until the job finished.
@@ -54,8 +51,9 @@ module Isono
       def run(endpoint, command, *args, &blk)
         cur_job_ctx = Thread.current[JobWorker::JOB_CTX_KEY]
         req = rpc.request("job.#{endpoint}", command, *args) { |req|
-          req.request[:job_request_type]=:run
-
+          req.timeout = config_section.timeout_sec
+          req.request[:job_id] = Util.gen_id
+          
           # A job is working on this current thread if cur_job_ctx is
           # not nil. Let the new job know the current job ID
           # as its parent job ID.
@@ -69,11 +67,12 @@ module Isono
         blk ? req : req.wait
       end
       
-      def cancel()
+      def cancel(job_id)
       end
       
-      def register_endpoint(endpoint, app)
-        rpc.register_endpoint("job.#{endpoint}", Rack::Job.new(app, JobWorker.new(node)))
+      def register_endpoint(endpoint, app, opts={})
+        opts = {:concurrency=>config_section.concurrency}.merge(opts)
+        rpc.register_endpoint("job.#{endpoint}", Rack::Job.new(app, JobWorker.new(node)), {:prefetch=>opts[:concurrency]})
       end
 
       private

data/lib/isono/node_modules/job_worker.rb

@@ -8,9 +8,13 @@ module Isono
       include Logger
 
       JOB_CTX_KEY=:job_worker_ctx
+
+      config_section do |c|
+        c.concurrency = 1
+      end
       
       initialize_hook do
-        @thread_pool = ThreadPool.new(10)
+        @thread_pool = ThreadPool.new(config_section.concurrency.to_i)
         @active_jobs = {}
 
         RpcChannel.new(node).register_endpoint("job-stats.#{node.node_id}", proc { |req, res|
@@ -29,30 +33,17 @@ module Isono
 
       # Start a new long term job.
       #
-      # @param [String] parent_id Parent Job ID for new job
-      # @param [Proc,nil] run_cb
-      # @param [Proc,nil] fail_cb
-      # @yield The block used as run_cb
+      # @yield The block to setup JobContext object.
       # 
-      # @example Simply set run block as yield block and parent job ID.
-      #  run('xxxxxx') {
-      #    # do something
+      # @example Initialize JobContext within the block.
+      #  start { |ctx|
+      #    # setup JobContext
+      #    ctx.job_id = 'xxxx'
+      #    ctx.parent_job_id = 'yyyy'
       #  }
-      # @example Set proc{} to both run and fail block.
-      #  run(proc{
-      #        # do something
-      #      }, proc{
-      #        # do rollback on fail
-      #      })
-      def run(parent_id=nil, run_cb=nil, fail_cb=nil, &blk)
-        if run_cb.is_a?(Proc)
-          job = JobContext.new(run_cb, parent_id)
-          job.fail_cb = fail_cb if fail_cb.is_a?(Proc)
-        elsif blk
-          job = JobContext.new(blk, parent_id)
-        else
-          raise ArgumentError, "callbacks were not set propery"
-        end
+      def start(&blk)
+        job = JobContext.new()
+        blk.call(job)
         @active_jobs[job.job_id] = job
         rpc = RpcChannel.new(node)
 
@@ -87,19 +78,47 @@ module Isono
         job
       end
 
+      # Run the block/proc. This is simple utility method for start().
+      #
+      # @param [Proc,nil] run_cb
+      # @param [Proc,nil] fail_cb
+      #
+      # @example Run 
+      #  run {
+      #    puts "message"
+      #  }
+      # @example Use proc{} for setting both run and fail block.
+      #  run(proc{
+      #        # do something
+      #      }, proc{
+      #        # do rollback on fail
+      #      })
+      def run(run_cb=nil, fail_cb=nil, &blk)
+        if run_cb.is_a?(Proc)
+          start do |job|
+            job.run_cb = run_cb
+            job.fail_cb = fail_cb if fail_cb.is_a?(Proc)
+          end
+        elsif blk
+          start do |job|
+            job.run_cb = blk
+          end
+        end
+      end
+
       class JobContext < OpenStruct
         include Logger
-        attr_reader :stm, :run_cb
-        attr_accessor :fail_cb
+        attr_reader :stm
+        attr_accessor :run_cb, :fail_cb
         
-        def initialize(run_cb, parent_id=nil)
+        def initialize()
           super({:job_id=>Util.gen_id,
-                  :parent_job_id=> parent_id,
+                  :parent_job_id=>nil,
                   :started_at=>nil,
                   :finished_at=>nil,
                 })
 
-          @run_cb=run_cb
+          @run_cb=proc{}
           @fail_cb=nil
 
           @stm = Statemachine.build {

data/lib/isono/node_modules/rpc_channel.rb

@@ -9,8 +9,6 @@ module Isono
     class RpcChannel < Base
       include Logger
 
-      AMQP_EXCHANGE='isono.rpc'
-
       class RpcError < RuntimeError; end
       class UnknownEndpointError < RpcError; end
       class DuplicateEndpointError < RpcError; end
@@ -23,14 +21,12 @@ module Isono
       initialize_hook do
         @active_requests = {}
         @endpoints = {}
-        amq.direct(AMQP_EXCHANGE, {:auto_delete=>true})
-        amq.queue("command-recv.#{manifest.node_id}", {:exclusive=>true}).subscribe { |header, data|
-          event = EventChannel.new(self.node)
+        @amq = node.create_channel
+        @amq.queue("command-recv.#{manifest.node_id}", {:exclusive=>true}).subscribe { |header, data|
           req = @active_requests[header.message_id]
           if req
             data = Serializer.instance.unmarshal(data)
             req.process_event(:on_received, data)
-            event.publish('rpc/response_received', :args=>[header.message_id])
             begin
               case data[:type]
               when :inprogress
@@ -69,9 +65,11 @@ module Isono
         @endpoints.keys.each { |ns|
           myinstance.unregister_endpoint(ns)
         }
-        amq.queue("command-recv.#{manifest.node_id}", {:exclusive=>true}).delete
+        @amq.queue("command-recv.#{manifest.node_id}", {:exclusive=>true}).delete
       end
 
+      attr_reader :amq
+
       # Make a RPC request to an endpoint.
       # 
       # @param [String] endpoint
@@ -112,24 +110,11 @@ module Isono
           # async 
           r = blk.call(req)
           req = r if r.is_a?(RequestContext)
-          if req.oneshot
-            send_request(req)
-          else
-            check_endpoint(endpoint) { |result|
-              if result
-                send_request(req)
-              else
-                e = UnknownEndpointError.new(endpoint)
-                req.error_cb.call(e) if req.error_cb
-              end
-            }
-          end
-
+          send_request(req)
           req
         else
           # sync
           req = req.synchronize
-          check_endpoint(endpoint) || raise(UnknownEndpointError, endpoint)
           send_request(req)
           req.wait
         end
@@ -141,21 +126,21 @@ module Isono
       # @param [String] endpoint
       # @param [has call() method]  app
       # @param [Hash] opts
+      #    :exclusive
+      #    :prefetch
       def register_endpoint(endpoint, app, opts={})
         raise TypeError unless app.respond_to?(:call)
-        opts = {:exclusive=>true}.merge(opts)
-        @endpoints[endpoint]={:app=>app, :opts=>opts}
+        opts = {:exclusive=>true, :prefetch=>0}.merge(opts)
         
         # create receive queue for new RPC endpoint.
         endpoint_proc = proc { |header, data|
 
           data = Serializer.instance.unmarshal(data)
-          event.publish('rpc/request_received', :args=>[header.message_id])
 
           resctx = if data[:oneshot]
-                     OneshotResponseContext.new(self.node, header)
+                     OneshotResponseContext.new(@endpoints[endpoint][:ch].response_exchange, header)
                    else
-                     ResponseContext.new(self.node, header)
+                     ResponseContext.new(@endpoints[endpoint][:ch].response_exchange, header)
                    end
           begin
             req = Rack::Request.new({:sender=>header.reply_to['command-recv.'.size..-1],
@@ -163,81 +148,51 @@ module Isono
                                     }.merge(data))
             res = Rack::Response.new(resctx)
             ret = app.call(req, res)
-          rescue Exception => e
+          rescue ::Exception => e
             logger.error(e)
             resctx.response(e) unless resctx.responded?
           end
         }
 
-        setup_proc = proc {
-          amq.queue(endpoint_queue_name(endpoint), {:exclusive=>false, :auto_delete=>true}).bind(
-                    AMQP_EXCHANGE, {:key=>endpoint_queue_name(endpoint)}
-                    ).subscribe(:ack=>true, &endpoint_proc) 
-          event.publish('rpc/register', :args=>[endpoint])
-        }
-
-        dm = Util::DeferedMsg.new(30)
         
         EventMachine.schedule {
-          amq.queue(endpoint_queue_name(endpoint), {:exclusive=>false, :auto_delete=>true}).status { |messages, consumers|
-            if opts[:exclusive]
-              if consumers.to_i == 0
-                setup_proc.call
-                dm.success
-              else
-                dm.error(DuplicateEndpointError.new("Endpoint is already locked: #{endpoint}"))
-              end
-            else
-              setup_proc.call
-              dm.success
+          ch = if opts[:prefetch].to_i > 0
+                 # create per endpoint channel
+                 node.create_channel
+               else
+                 # use default channel
+                 @amq
+               end
+          ch.instance_eval %Q{
+            def endpoint_queue
+              self.queue("isono.rpc.endpoint.#{endpoint}", {:exclusive=>false, :auto_delete=>true})
             end
 
-            # expect to raise DuplicateEndpointError if endpoint exists.
-            # ignore the case of success.
-            dm.wait
+            def response_exchange
+              self.direct('')
+            end
           }
-        }
+          ch.prefetch(opts[:prefetch].to_i) if opts[:prefetch].to_i > 0
+          # stores hash here that is for thread safety.
+          @endpoints[endpoint]={:app=>app, :opts=>opts, :ch=>ch}
 
-        dm.wait unless EventMachine.reactor_thread?
+          ch.endpoint_queue.subscribe(:ack=>true, &endpoint_proc)
+          event.publish('rpc/register', :args=>[endpoint])
+        }
       end
 
       # Unregister endpoint.
       # 
       # @param [String] endpoint endpoint name to be removed
       def unregister_endpoint(endpoint)
-        if @endpoints.delete(endpoint)
-          dm = Util::DeferedMsg.new(30)
+        if @endpoints.has_key?(endpoint)
           EventMachine.schedule {
-            amq.queue(endpoint_queue_name(endpoint)).delete
+            data = @endpoints.delete(endpoint)
+            # endpoint_queue is :auto_delete=>true so that it will be deleted
+            # in case of zero consumers.
+            data[:ch].endpoint_queue.unsubscribe
             event.publish('rpc/unregister', :args=>[endpoint])
-            dm.success
           }
-          dm.wait unless EventMachine.reactor_thread?
-        end
-      end
-
-      # Check if the endpoint exists.
-      # @param [String] endpoint endpoint name to be checked
-      def check_endpoint(endpoint, &blk)
-        if blk
-        else
-          dm = Util::DeferedMsg.new(30)
-        end
-
-        EventMachine.schedule {
-          amq.queue(endpoint_queue_name(endpoint), {:exclusive=>false, :auto_delete=>true}).status { |messages, consumers|
-            res = consumers.to_i > 0
-            if blk
-              blk.call(res)
-            else
-              dm.success(res)
-            end
-          }
-        }
-        
-        if blk
-        else
-          dm.wait unless EventMachine.reactor_thread?
         end
       end
       
@@ -280,24 +235,22 @@ module Isono
           if !req.oneshot
             @active_requests[req.ticket] = req
           end
-          
-          amq.direct(AMQP_EXCHANGE).publish(
-                  Serializer.instance.marshal(req.request_hash),
-                  {:message_id => req.ticket,
-                    :key => endpoint_queue_name(req.endpoint),
-                    :reply_to=>"command-recv.#{manifest.node_id}"}
-                  )
+
+          amq.direct('').publish(Serializer.instance.marshal(req.request_hash),
+                       {:message_id => req.ticket,
+                         :key => endpoint_queue_name(req.endpoint),
+                         :reply_to=>"command-recv.#{manifest.node_id}"}
+                       )
           req.process_event(:on_sent)
-          event.publish('rpc/request_sent', :args=>[req.hash])
         }
       end
 
       class ResponseContext
-        attr_reader :node, :header
+        attr_reader :header
         
-        def initialize(node, header)
+        def initialize(exchange, header)
           @responded = false
-          @node = node
+          @exchange = exchange
           @header = header
         end
 
@@ -321,15 +274,13 @@ module Isono
             else
               publish(:success, ret)
             end
-            EventChannel.new(@node).publish('rpc/response_sent', :args=>[@header.message_id])
           }
           @responded = true
         end
 
-
         private
         def publish(type, body)
-          @node.amq.direct('').publish(Serializer.instance.marshal({:type=>type, :msg=>body}),
+          @exchange.publish(Serializer.instance.marshal({:type=>type, :msg=>body}),
                                        {:key=>@header.reply_to,
                                          :message_id=>@header.message_id}
                                        )
@@ -347,7 +298,6 @@ module Isono
 
           EM.schedule {
             @header.ack
-            EventChannel.new(@node).publish('rpc/response_sent', :args=>[@header.message_id])
           }
           @responded = true
         end